#!/usr/bin/env python
# -*- coding: utf8 -*-

'''
Created on 2011-8-25

@author: zhongfeng

'''

from icson.icsonpageparser import parserDict,icsonRoot
from spider import ObuySpider
from crawlerhttp import CrawlResult
import os,sys
from logfacade import LoggerFactory
from upload import fileUpload
from spiderconfigparser import SpiderConfig

def main(fileName):
    with open(fileName, 'r') as fInput:
        content = fInput.read()
    rootResult = CrawlResult(code=200, content=content)
    from spiderconfigparser import getIncludeUrlSums,getExcludeUrlSums
    includes = getIncludeUrlSums()
    excludes = getExcludeUrlSums()
    spider = ObuySpider(rootUrlSummary=icsonRoot, parserDict=parserDict,include=includes,exclude = excludes,
                                     rootPageResult=rootResult, threadNum=SpiderConfig.getThreadNum())
    if SpiderConfig.isStartSpider():
        spider.spide()
    LoggerFactory.shutdown()
    if SpiderConfig.isUpload():
        fileUpload()

if __name__ == '__main__':
    curPath = os.path.abspath(os.path.dirname(sys.argv[0]))
    fileName = os.path.join(curPath, 'portal.html')
    main(fileName)
